import json
import datetime
import os
from collections import Counter
import csv
import time
import statistics
import numpy
from datetime import datetime
from datetime import timedelta
# ht=[ '#YearOnTikTok', '#lovestory','#artmas','#festivefashion','#happyhanukkah','#planttiktok','#productivity','#ImAGhost','#catchphrases','#ootd','#comfortfood','#holidaycrafts','#watchmegrow','#holidaydecor','#raisedby','#makeitvogue','#goodmorning','#watchingxfactor','#catchphrases','#rnbvibes','#gamenight','#HorrorToK', '#ProblemSolved', '#SkateLife', '#ThriftShop', '#SpookyTreats',  '#tiktokfood', '#interiordesign', '#sfxmakeup', '#ComingOfAge', '#falldiy', '#worldseries', '#gaminglife', '#meleaving', '#yellow', '#yougotthis','#mycostume','#mypfp','#welldone','#nativefamily','#happyhalloween','#homeoffice','#myrecommendation','#myhobby','#fanedit','#stemlife','#motivationmonday','#wip','#veteransday','#holidaycountdown','#WeekendVibes','#holidaysourway','#RoomTour','#bakingszn','#nbadraft',
#      '#nonuancenovember','#gamingsetup','#carsoftiktok','#needtoknow','#whenwewereyounger','#halloweenishere','#ourtype','#fallguysmoments','#artmas','#onlinedating','#familyrecipe','#givingthanks','#diceroll','#holidayvibes','#holidaytiktok','#recordsday','#inkdrawing','#happyholidays','#homecooked','#neonshadow',  '#youwantmore' ,'#familyimpression','#givingszn','#whereilive','foodtiktok','#holidaymusic','#howbizarre', '#personalfinance'
#      ,'#thinkingabout', '#housetour','#readysetshop','#easydiy','#coldweather','#wildanimals','#festivefashion','graphicdesign','smallbusiness','#halloweenlook','fallfashion','#artmas','#haventseen','#planttiktok','#productivity','#bekind','#onhold','#growupwithme','#tabletop','#theatrekids','#wishlist','#haventseen','#selfimprovement', '#OhNo','#animation']
# ht=['#festivefashion', '#selfimprovement','#happyhanukkah','#planttiktok','#productivity','#ImAGhost','#catchphrases','#ootd','#comfortfood','#holidaycrafts','#watchmegrow','#holidaydecor','#raisedby','#makeitvogue','#goodmorning','#catchphrases','#rnbvibes','#gamenight','#HorrorToK', '#yougotthis','#mycostume','#nbaisback','#perfectgifts','#YearOnTikTok','#artmas','#hellowinter','#Wishlist','#haventseen','#RoomTour','#bakingszn',
#    '#christmas2020','#feliznavidad','#timewarpjump','#wrappinggifts','#holidaytreats','#nonuancenovember','#gamingsetup','#carsoftiktok','#needtoknow','#whenwewereyounger','#halloweenishere','#ourtype','#fallguysmoments','#artmas','#onlinedating','#familyrecipe','#givingthanks','#diceroll','#holidayvibes','#holidaytiktok','#recordsday','#inkdrawing','#happyholidays','#homecooked','#neonshadow',  '#youwantmore' ,'#familyimpression','#givingszn','#whereilive','foodtiktok', '#personalfinance', '#tiktokfood'
#   ,'#thinkingabout','#winterfit','#withouttellingme','#myhaul','#Bye2020','#welcome2021','#rareaesthetic', '#hyperfixated','#whatilearned','#dailyvlog','#easyrecipe','#housetour','#readysetshop','#easydiy','#wildanimals','#festivefashion','graphicdesign','#mystyle','#halloweenlook','#bekind','#onhold','smallbusiness','#coldweather','#growupwithme','#selfimprovement', '#nbadraft','#lovestory','fallfashion','#planttiktok','#OhNo','#animation','#tabletop','#homeoffice','#myrecommendation',
#     '#RatatouilleMusical','#myhobby','#fanedit','#stemlife', '#interiordesign','#holidaymusic','#howbizarre','#motivationmonday','#productivity','#wip','#veteransday', '#sfxmakeup', '#ComingOfAge', '#falldiy', '#gaminglife', '#meleaving', '#yellow','#welldone','#nativefamily','#holidaycountdown','#WeekendVibes','#WordsOfWisdom','#happyhalloween','#mypfp','#cozyathome', '#worldseries','#holidaysourway','#theatrekids']
#,'#myroutine'
ht=['ComingOfAge', 'falldiy', 'gaminglife', 'interiordesign', 'oneyourthere', 'sfxmakeup', 'tiktokfood', 'worldseries', 'halloweenishere', 'meleaving', 'yellow', 'yougotthis', 'mycostume', 'animation', 'youwantmore', 'halloweenlook', 'mypfp', 'nativefamily', 'welldone', 'happyhalloween', 'OhNo', 'holidaytiktok', 'rnbvibes', 'carsoftiktok', 'homeoffice', 'myrecommendation', 'fallfashion', 'familyimpression', 'myhobby', 'neonshadow', 'fanedit', 'stemlife', 'motivationmonday', 'wip', 'veteransday', 'holidaycountdown', 'whereilive', 'bekind', 'onhold', 'diwali', 'WeekendVibes', 'bakingszn', 'RoomTour', 'foodtiktok', 'growupwithme', 'holidaysourway', 'tabletop', 'nbadraft', 'recordsday', 'ourtype', 'nonuancenovember', 'artmas', 'gamingsetup', 'theatrekids', 'needtoknow', 'whenwewereyounger', 'graphicdesign', 'inkdrawing', 'fallguysmoments', 'familyrecipe', 'onlinedating', 'watchmegrow', 'givingthanks', 'readysetshop', 'diceroll', 'smallbusiness', 'holidayvibes', 'coldweather', 'givingszn', 'happyholidays', 'homecooked', 'easydiy', 'wildanimals', 'holidaymusic', 'howbizarre', 'selfimprovement', 'ImAGhost', 'personalfinance', 'housetour', 'thinkingabout', 'holidaycrafts', 'happyhanukkah', 'RatatouilleMusical', 'catchphrases', 'ootd', 'comfortfood', 'holidaydecor', 'raisedby', 'goodmorning', 'makeitvogue', 'festivefashion', 'gamenight', 'artmas', 'haventseen', 'planttiktok', 'productivity', 'lovestory', 'wishlist', 'hellowinter', 'YearOnTikTok', 'nbaisback', 'perfectgifts', 'holidaytreats', 'wrappinggifts', 'feliznavidad', 'timewarpjump', 'christmas2020', 'MyHaul', 'cozyathome', 'myhaul', 'winterfit', 'withouttellingme', 'hyperfixated', 'whatilearned',  'WordsOfWisdom',  'rareaesthetic', 'welcome2021', 'dailyvlog', 'easyrecipe', 'mystyle', 'Bye2020', 'myroutine', 'gamergoals', 'projectcar', 'homemade', 'inlove', 'GreenScreenScan', 'tortillatrend', 'NFLplayoffs', 'FitnessRoutine', 'IsThisAvailable', 'wee', 'weirdpets', 'fitnessroutine',  'moneytok', 'nflplayoffs', 'healthycooking', 'nhlfaceoff', 'groupchat', 'NHLFaceOff', 'winterfashion', 'skincare101', 'zodiacsign', 'homeimprovement', 'seashanty', 'cleantok', 'visionboard', '2021affirmations', 'mlkday', 'joblife', 'foodie', 'timewarpwaterfall', 'plantparent', 'WinterMagic', 'notaperfectperson', 'tiktoktutorial', 'cocinando', 'OlympicsCountdown', 'couplethings', 'clingypet', 'meditation101', 'winterbeautytips', 'tiktokdiy', 'typing', 'roundofapplause', 'xgamesmode', 'feelinggood', 'RoyalRumble', 'emophase', 'wintersports', 'makeblackhistory', 'relationshipstorytime', 'albumcover', 'stepbystep', 'fetapasta', 'womeninsports', 'healthyheart', 'imbusyrightnow', 'beautyhacks', 'tiktoktailgate', 'puppybowl', 'superbowllv', 'valentinesdiy', 'melaninmagic', 'coversforlovers', 'kissyourpet', 'womeninstem', 'lunarnewyear', 'perfectmatch', 'galentinesday', 'valentinesday', 'loveyourinsecurities', 'tiktokfashionmonth', 'blackcreatives', 'colddays', 'mifamilia', 'careeradvice', 'stopasianhate', 'perfectdrink', 'snowstorm', 'carhacks', 'homeproject', 'blackandproud', 'dramaticmoments', 'bakedoats', 'homecook', 'laughingduet', 'yoga101', 'somethingyoulearned', 'upcycling', 'fantheory', 'tiktokfitness', 'gamingtiktok', 'seitan', 'glasspainting', 'whenwomenwin', 'science101', 'thriftflip', 'Lifestyle', 'tiktokwildlifeday', 'dayandnight']
#ht=['ComingOfAge', 'falldiy', 'gaminglife', 'interiordesign', 'oneyourthere', 'sfxmakeup', 'tiktokfood', 'worldseries', 'halloweenishere', 'meleaving']
#ht=['makeitvogue','LGBTQHistory','wip','haventseen','holidayvibes','SkateLife','ootd','bekind','personalfinance','cozyathome','gamenight','happyholidays','ProblemSolved','veteransday','RoomTour','meleaving','yellow']
# for name in os.listdir('D:\\Work\\Tool\\tiktok\\TikToks\\'):
#     if not(name.endswith('.json')):
#         continue
#     if not(name.lower().startswith('hashtag')):
#         continue
#     fname=name.replace('.json','').split('_')
#     if fname[1].lower() not in ht:
#         ht.append(fname[1].lower())
hts=[h.lower().replace('#','') for h in ht]
ht=[]
for item in hts:
    if item not in ht:
        ht.append(item)
for l in range(len(ht)):
    print(ht[l],l+1)
htranking=0
ddt=datetime(2021, 4, 24)
with open('D:\\Work\\Tool\\tiktok\\all\\diggcount_d7'+'.tsv', 'w', encoding='utf-8',
              newline='\n') as filename_output:
    writerp = csv.writer(filename_output, delimiter='\t')
    with open('D:\\Work\\Tool\\tiktok\\all\\days_d7'+ '.tsv', 'w', encoding='utf-8',
              newline='\n') as filename_output:
        writerd = csv.writer(filename_output, delimiter='\t')
        with open('D:\\Work\\Tool\\tiktok\\all\\ab_d7' + '.tsv', 'w', encoding='utf-8',
                  newline='\n') as filename_output2:
            writerab = csv.writer(filename_output2, delimiter='\t')
            writerab.writerow(['hashtagid', 'id', 'a', 'b', 'loga', 'logb'])
            for hashtag in ht:
                print(hashtag)
                htranking+=1
                c=Counter()
                datalist=[]
                dict = {}
                pdict={}
                ddict={}
                for name in os.listdir('D:\\Work\\Tool\\tiktok\\TikToks\\'):
                    htname = name.split('_')[1]
                    if hashtag in htname.lower():
                        dt = name.split('_')[2].replace('.json', '')
                        if ddt < datetime.strptime(dt, '%Y-%m-%d'):
                            continue

                        datalist.append(name)
                    # print(row)
                for i in range(len(datalist)):
                    ranking=0
                    with open('D:\\Work\\Tool\\tiktok\\TikToks\\' + datalist[i], 'r', encoding='utf-8', newline='\n') as filename_input:
                        f=[]
                        for line in filename_input:

                            z = json.loads(line)
                            if 'id' in z.keys():
                                id=z['id']
                                pc=int(z['stats']['playCount'])
                                # if time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['createTime'])))<dict[hashtag][0]:
                                #     dict[hashtag][0]=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['createTime'])))
                                # f=False
                                # for item in z['challenges']:
                                #     if hashtag in item['title'].lower():
                                #         f=True
                                #         break
                                # if time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['createTime'])))<dict[hashtag][1] and f :
                                #     dict[hashtag][1]=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['createTime'])))
                            elif 'itemInfos' in z.keys():
                                id=z['itemInfos']['id']
                                pc=int(z['itemInfos']['playCount'])
                                # if time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['itemInfos']['createTime'])))<dict[hashtag][0]:
                                #     dict[hashtag][0] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['itemInfos']['createTime'])))
                                # f = False
                                # for item in z["challengeInfoList"]:
                                #     if hashtag in item["challengeName"].lower():
                                #         f = True
                                #         break
                                # if time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['itemInfos']['createTime']))) < dict[hashtag][1] and f:
                                #     dict[hashtag][1] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(z['itemInfos']['createTime'])))
                            ranking += 1
                            if ranking > 2000:
                                break

                            if id in f:
                                continue
                            else:
                                f.append(id)
                            following = int(z['authorStats']['followingCount'])
                            follower = int(z['authorStats']['followerCount'])
                            heart = int(z['authorStats']['heartCount'])
                            video = int(z['authorStats']['videoCount'])
                            digg = int(z['authorStats']['diggCount'])
                            if following <= 0 or follower <= 0 or pc <= 0 or video <= 0 or digg <= 0 or heart <=0:
                                continue
                            if pc ==0:
                                continue

                            if id in dict.keys():
                                dict[id].append(ranking)
                                pdict[id].append(pc)
                                ddict[id].append(i+1)
                            else:
                                dict[id]=[]
                                pdict[id] = []
                                ddict[id]=[]
                                # for k in range(i):
                                #     dict[id].append('NA')
                                #     # pdict[id].append('NA')
                                dict[id].append(ranking)
                                pdict[id].append(pc)
                                ddict[id].append(i+1)
                        # for key in dict.keys():
                        #     if len(dict[key])<=i:
                        #         dict[key].append('NA')
                                # pdict[key].append('NA')
                j=0


                # row=['id']
                #                 # row.extend(datalist)
                #                 # # print(len(row))
                #                 # writerp.writerow(row)
                idlist=[]

                for key in ddict.keys():
                    if len(ddict[key]) <= 1:
                        continue
                    rows = [htranking, key]
                    m = ddict[key][0] - 1
                    a = numpy.array([item - m for item in ddict[key][:30]])
                    b = numpy.array(pdict[key][:30])
                    rows.extend(list(numpy.polyfit(numpy.log(a), b, 1)))
                    rows[2] = rows[2]
                    rows[3] = rows[3]
                    if rows[2] <= 0 or rows[3] <= 0:
                        continue
                    rows.append(numpy.log(rows[2]))
                    rows.append(numpy.log(rows[3]))
                    writerab.writerow(rows)
                    idlist.append(key)
                j=0
                for key in pdict.keys():
                    if key not in idlist:
                        continue
                    rows=[htranking,key]
                    rows.extend(pdict[key])

                    if len(pdict[key])<=1:
                        continue
                    writerp.writerow(rows)
                j = 0

                for key in ddict.keys():
                    if key not in idlist:
                        continue
                    rows = [htranking,key]
                    #
                    m=ddict[key][0]-1
                    # for item in ddict[key]:
                    #     copy.append(item-m)
                    rows.extend([item-m for item in ddict[key]])
                    if len(ddict[key])<=1:
                        continue
                    writerd.writerow(rows)
                j=0


        # for key in ddict.keys():
        #     rows = [key]
        #     rows.extend(ddict[key])
        #     if j < 100:
        #         print(len(rows))
        #         print(rows)
        #         j += 1
        #     writer.writerow(rows)
        # vv=[]
        # for key in dict.keys():
        #     fline=[]
        #     i=0
        #     for j in range(i,len(dict[key])):
        #         if dict[key][j]!='NA':
        #             break
        #         else:
        #             i+=1
        #     for j in range(i,len(dict[key])):
        #         if dict[key][j]!='NA':
        #             fline.append(int(dict[key][j]))
        #             i+=1
        #         else:
        #             break
        #     j=0
        #     if len(fline)<=1:
        #         rows = [key, 'NA']
        #         rows.extend(fline)
        #     else:
        #         rows=[key,round(statistics.variance(fline),4)]
        #         vv.append(statistics.variance(fline))
        #         rows.extend(fline)
        #     # if j<100:
        #     #     print(len(rows))
        #     #     print(rows)
        #     #     j+=1
        #     writer.writerow(rows)
        # print('Variance for ',hashtag,statistics.mean(vv))
        # for key in pdict.keys():
        #     rows=[key]
        #     rows.extend(pdict[key])
        #     if j<100:
        #         print(len(rows))
        #         print(rows)
        #         j+=1
        #     writer.writerow(rows)


    # dict[hashtag] = [datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'),
    #                  datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), datalist[0].split('_')[2]]
    #
    # print(hashtag,dict[hashtag][0],dict[hashtag][1],dict[hashtag][2])
    # writer.writerow([hashtag,dict[hashtag][0],dict[hashtag][1],dict[hashtag][2]])